# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import pymongo, json, requests
from xinzhuasearch import settings
from xinzhuasearch.untils.utils_log import log_out, log

class XinzhuasearchPipeline(object):
    def __init__(self):
        # 链接数据库
        client = pymongo.MongoClient(host=settings.MONGO_HOST, port=settings.MONGO_PORT)
        self.db = client[settings.MONGO_DB]  # 获得数据库的句柄
        self.coll = self.db[settings.MONGO_COLL]  # 获得collection的句柄
        # 数据库登录需要帐号密码的话
        # self.db.authenticate(settings.MONGO_USER, settings.MONGO_PSW)

    def process_item(self, item, spider):
        postItem = dict(item)  # 把item转化成字典形式
        self.coll.insert(postItem)  # 向数据库插入一条记录
        return item


class JsonPipeline(object):
    def __init__(self):
        self.file = open('seo.json', 'wb')
        self.datas = []

    def process_item(self, item, spider):
        if len(self.datas) < 20:
            self.datas.append(dict(item))
        else:
            self.datas.append(dict(item))
            line = json.dumps(self.datas, ensure_ascii=False) + "\n"
            self.file.write(line.encode('utf-8'))
            self.datas = []
        return item

    def spider_closed(self, spider):
        self.file.close()